3 * @defgroup ExternalStorage ExternalStorage
6 use MediaWiki\MediaWikiServices
;
9 * Interface for data storage in external repositories.
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License along
22 * with this program; if not, write to the Free Software Foundation, Inc.,
23 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
24 * http://www.gnu.org/copyleft/gpl.html
30 * Constructor class for key/value blob data kept in external repositories.
32 * Objects in external stores are defined by a special URL. The URL is of
33 * the form "<store protocol>://<location>/<object name>". The protocol is used
34 * to determine what ExternalStoreMedium class is used. The location identifies
35 * particular storage instances or database clusters for store class to use.
37 * When an object is inserted into a store, the calling code uses a partial URL of
38 * the form "<store protocol>://<location>" and receives the full object URL on success.
39 * This is useful since object names can be sequential IDs, UUIDs, or hashes.
40 * Callers are not responsible for unique name generation.
42 * External repositories might be populated by maintenance/async
43 * scripts, thus partial moving of data may be possible, as well
44 * as the possibility to have any storage format (i.e. for archives).
46 * @ingroup ExternalStorage
50 * Get an external store object of the given type, with the given parameters
52 * @param string $proto Type of external storage, should be a value in $wgExternalStores
53 * @param array $params Associative array of ExternalStoreMedium parameters
54 * @return ExternalStoreMedium|bool The store class or false on error
56 public static function getStoreObject( $proto, array $params = [] ) {
57 return MediaWikiServices
::getInstance()
58 ->getExternalStoreFactory()
59 ->getStoreObject( $proto, $params );
63 * Fetch data from given URL
65 * @param string $url The URL of the text to get
66 * @param array $params Associative array of ExternalStoreMedium parameters
67 * @return string|bool The text stored or false on error
70 public static function fetchFromURL( $url, array $params = [] ) {
71 $parts = explode( '://', $url, 2 );
72 if ( count( $parts ) != 2 ) {
73 return false; // invalid URL
76 list( $proto, $path ) = $parts;
77 if ( $path == '' ) { // bad URL
81 $store = self
::getStoreObject( $proto, $params );
82 if ( $store === false ) {
86 return $store->fetchFromURL( $url );
90 * Fetch data from multiple URLs with a minimum of round trips
92 * @param array $urls The URLs of the text to get
93 * @return array Map from url to its data. Data is either string when found
94 * or false on failure.
96 public static function batchFetchFromURLs( array $urls ) {
98 foreach ( $urls as $url ) {
99 $scheme = parse_url( $url, PHP_URL_SCHEME
);
101 $batches[$scheme][] = $url;
105 foreach ( $batches as $proto => $batchedUrls ) {
106 $store = self
::getStoreObject( $proto );
107 if ( $store === false ) {
110 $retval +
= $store->batchFetchFromURLs( $batchedUrls );
112 // invalid, not found, db dead, etc.
113 $missing = array_diff( $urls, array_keys( $retval ) );
115 foreach ( $missing as $url ) {
116 $retval[$url] = false;
124 * Store a data item to an external store, identified by a partial URL
125 * The protocol part is used to identify the class, the rest is passed to the
126 * class itself as a parameter.
128 * @param string $url A partial external store URL ("<store type>://<location>")
129 * @param string $data
130 * @param array $params Associative array of ExternalStoreMedium parameters
131 * @return string|bool The URL of the stored data item, or false on error
132 * @throws MWException
134 public static function insert( $url, $data, array $params = [] ) {
135 $parts = explode( '://', $url, 2 );
136 if ( count( $parts ) != 2 ) {
137 return false; // invalid URL
140 list( $proto, $path ) = $parts;
141 if ( $path == '' ) { // bad URL
145 $store = self
::getStoreObject( $proto, $params );
146 if ( $store === false ) {
149 return $store->store( $path, $data );
154 * Like insert() above, but does more of the work for us.
155 * This function does not need a url param, it builds it by
156 * itself. It also fails-over to the next possible clusters
157 * provided by $wgDefaultExternalStore.
159 * @param string $data
160 * @param array $params Associative array of ExternalStoreMedium parameters
161 * @return string|bool The URL of the stored data item, or false on error
162 * @throws MWException
164 public static function insertToDefault( $data, array $params = [] ) {
165 global $wgDefaultExternalStore;
167 return self
::insertWithFallback( (array)$wgDefaultExternalStore, $data, $params );
171 * Like insert() above, but does more of the work for us.
172 * This function does not need a url param, it builds it by
173 * itself. It also fails-over to the next possible clusters
174 * as provided in the first parameter.
176 * @param array $tryStores Refer to $wgDefaultExternalStore
177 * @param string $data
178 * @param array $params Associative array of ExternalStoreMedium parameters
179 * @return string|bool The URL of the stored data item, or false on error
180 * @throws MWException
182 public static function insertWithFallback( array $tryStores, $data, array $params = [] ) {
184 while ( count( $tryStores ) > 0 ) {
185 $index = mt_rand( 0, count( $tryStores ) - 1 );
186 $storeUrl = $tryStores[$index];
187 wfDebug( __METHOD__
. ": trying $storeUrl\n" );
188 list( $proto, $path ) = explode( '://', $storeUrl, 2 );
189 $store = self
::getStoreObject( $proto, $params );
190 if ( $store === false ) {
191 throw new MWException( "Invalid external storage protocol - $storeUrl" );
194 $url = $store->store( $path, $data ); // Try to save the object
195 } catch ( Exception
$error ) {
198 if ( strlen( $url ) ) {
199 return $url; // Done!
201 unset( $tryStores[$index] ); // Don't try this one again!
202 $tryStores = array_values( $tryStores ); // Must have consecutive keys
203 wfDebugLog( 'ExternalStorage',
204 "Unable to store text to external storage $storeUrl" );
209 throw $error; // rethrow the last error
211 throw new MWException( "Unable to store text to external storage" );
216 * @param string $data
217 * @param string $wiki
218 * @return string|bool The URL of the stored data item, or false on error
219 * @throws MWException
221 public static function insertToForeignDefault( $data, $wiki ) {
222 return self
::insertToDefault( $data, [ 'wiki' => $wiki ] );